{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6bfa314b-cc99-4d59-a281-94bfcb49b289",
   "metadata": {},
   "outputs": [],
   "source": [
    "!pip install nbformat==5.1.3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c42b885f-2551-495a-8deb-4f7f6e755db8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import configparser\n",
    "import os"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7461ab98-07c3-49b8-b648-a1cfc383de62",
   "metadata": {},
   "outputs": [],
   "source": [
    "config = configparser.ConfigParser()\n",
    "config.read('ibm-sql-query.secrets')\n",
    "config  = config['DEFAULT']\n",
    "for key in config:  \n",
    "    exec('{}=\"{}\"'.format(key,config[key]))\n",
    "    os.environ[key] = config[key]\n",
    "\n",
    "config = configparser.ConfigParser()\n",
    "config.read('ibm-sql-query.config')\n",
    "config  = config['DEFAULT']\n",
    "for key in config:  \n",
    "    exec('{}=\"{}\"'.format(key,config[key]))\n",
    "    os.environ[key] = config[key]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c64a2466-2c43-43ed-98b1-f7dd5d1b5ca2",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "# delete result if existing\n",
    "if [ `ipython cloud-object-store-housekeeping.ipynb 2> /dev/null | grep result |wc -l` -gt 0 ]; then\n",
    "    ipython cloud-object-store-housekeeping.ipynb operation=\"rm\" path=\"result\"\n",
    "fi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d51ee3e0-e16a-4236-9c59-1067a256b898",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "# verify result deleted\n",
    "if [ `ipython cloud-object-store-housekeeping.ipynb 2> /dev/null | grep result |wc -l` -gt 0 ]; then\n",
    "    echo 'deletion failed'\n",
    "    exit -1\n",
    "fi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5c13f791-7bb6-46f1-9d01-31363235259d",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "!ipython ibm-sql-query.ipynb"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f71d5ba-7d48-41a9-9253-5bee19f166f9",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "# verify result existing\n",
    "if [ `ipython cloud-object-store-housekeeping.ipynb 2> /dev/null | grep result |wc -l` -eq 0 ]; then\n",
    "    echo 'creation failed'\n",
    "    exit -1\n",
    "fi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9af7177f-b171-46e1-96d2-8df78038cf30",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "# delete result\n",
    "if [ `ipython cloud-object-store-housekeeping.ipynb 2> /dev/null | grep result |wc -l` -gt 0 ]; then\n",
    "    ipython cloud-object-store-housekeeping.ipynb operation=\"rm\" path=\"result\"\n",
    "fi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a788ed9f-b9a6-4073-9ed9-b870a4081624",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "# verify result deleted\n",
    "if [ `ipython cloud-object-store-housekeeping.ipynb 2> /dev/null | grep result |wc -l` -gt 0 ]; then\n",
    "    echo 'deletion failed'\n",
    "    exit -1\n",
    "fi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0f1ecb54-384e-48bc-9b49-045403bcd47f",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "!ipython ibm-sql-query.ipynb out_format='parquet'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "66a1449c-25d7-4bed-9194-5296ae2410bd",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "# verify result existing\n",
    "if [ `ipython cloud-object-store-housekeeping.ipynb 2> /dev/null | grep result |wc -l` -eq 0 ]; then\n",
    "    echo 'creation failed'\n",
    "    exit -1\n",
    "fi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "c080cb30-2f8d-43a7-b0ed-a08c390c4e62",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "# delete result\n",
    "if [ `ipython cloud-object-store-housekeeping.ipynb 2> /dev/null | grep result |wc -l` -gt 0 ]; then\n",
    "    ipython cloud-object-store-housekeeping.ipynb operation=\"rm\" path=\"result\"\n",
    "fi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "84369b66-35b1-4509-ad0b-860684069e26",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "# verify result deleted\n",
    "if [ `ipython cloud-object-store-housekeeping.ipynb 2> /dev/null | grep result |wc -l` -gt 0 ]; then\n",
    "    echo 'deletion failed'\n",
    "    exit -1\n",
    "fi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ebc20cc-0b8f-4e1b-b9ba-7e7d4db87f4f",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "!ipython ibm-sql-query.ipynb out_partition_columns='x,y,z'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6005f6ec-f2f3-4f51-8e0c-45d1032adbe7",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "# verify result existing and corrent number of partitions\n",
    "if [ `ipython cloud-object-store-housekeeping.ipynb operation=\"walk\" path=\"result\" 2> /dev/null |grep result |wc -l` -lt 1000 ]; then\n",
    "    echo 'creation failed, number of partitions < 1000'\n",
    "    exit -1\n",
    "fi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bee20853-2578-4915-af89-31e814a4cb36",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "# delete result\n",
    "if [ `ipython cloud-object-store-housekeeping.ipynb 2> /dev/null | grep result |wc -l` -gt 0 ]; then\n",
    "    ipython cloud-object-store-housekeeping.ipynb operation=\"rm\" path=\"result\"\n",
    "fi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2934f4a0-8ed8-491b-8f3c-8c7d27810210",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "# verify result deleted\n",
    "if [ `ipython cloud-object-store-housekeeping.ipynb 2> /dev/null | grep result |wc -l` -gt 0 ]; then\n",
    "    echo 'deletion failed'\n",
    "    exit -1\n",
    "fi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "df8072ed-7198-4b5d-9586-7ea546785c8a",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "!ipython ibm-sql-query.ipynb out_partition_columns='x,y,z' out_format='parquet'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "860cdb15-9736-4374-b6ed-9aca6f651a4d",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "# verify result existing and corrent number of partitions\n",
    "if [ `ipython cloud-object-store-housekeeping.ipynb operation=\"walk\" path=\"result\" 2> /dev/null |grep result |wc -l` -lt 1000 ]; then\n",
    "    echo 'creation failed, number of partitions < 1000'\n",
    "    exit -1\n",
    "fi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4ad716d5-8fe9-41f7-b9df-7c2f91f99f69",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "# delete result\n",
    "if [ `ipython cloud-object-store-housekeeping.ipynb 2> /dev/null | grep result |wc -l` -gt 0 ]; then\n",
    "    ipython cloud-object-store-housekeeping.ipynb operation=\"rm\" path=\"result\"\n",
    "fi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0048e10b-65d6-465f-9d88-0abc7076a79c",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "# verify result deleted\n",
    "if [ `ipython cloud-object-store-housekeeping.ipynb 2> /dev/null | grep result |wc -l` -gt 0 ]; then\n",
    "    echo 'deletion failed'\n",
    "    exit -1\n",
    "fi"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9961c5b8-932b-45f4-a256-e8cdc85592fd",
   "metadata": {
    "tags": []
   },
   "source": [
    "# TEST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b655d0f4-4ced-4042-a357-9a32d348f714",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "!ipython ibm-sql-query.ipynb out_format='parquet' out_partition_columns='x,y,z' out_number_of_objects=\"42\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "67e5e80a-01ad-469b-96c5-5dfec353d0ce",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "# verify result existing and corrent number of partitions\n",
    "if [ `ipython cloud-object-store-housekeeping.ipynb operation=\"walk\" path=\"result\" 2> /dev/null |grep result |wc -l` -ne 2 ]; then\n",
    "    echo 'creation failed, number of partitions <> 2'\n",
    "    exit -1\n",
    "fi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8557895e-af81-49c6-ac30-a2c422756091",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "# delete result\n",
    "if [ `ipython cloud-object-store-housekeeping.ipynb 2> /dev/null | grep result |wc -l` -gt 0 ]; then\n",
    "    ipython cloud-object-store-housekeeping.ipynb operation=\"rm\" path=\"result\"\n",
    "fi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "90aba535-edcb-452a-97b4-6d18a2bb2de0",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "# verify result deleted\n",
    "if [ `ipython cloud-object-store-housekeeping.ipynb 2> /dev/null | grep result |wc -l` -gt 0 ]; then\n",
    "    echo 'deletion failed'\n",
    "    exit -1\n",
    "fi"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "0a3d14b0-a4fe-492d-85e3-af807f888a6b",
   "metadata": {
    "tags": []
   },
   "source": [
    "# TEST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b2789ab7-f7ad-4089-9c4e-5bd96f898361",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "!ipython ibm-sql-query.ipynb out_format='parquet' out_partition_columns='x,y,z' out_number_of_objects=\"42\" out_rows_per_object=\"23\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf0cbb88-784f-4bb3-a25d-b79660883552",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "# verify result existing and corrent number of partitions\n",
    "if [ `ipython cloud-object-store-housekeeping.ipynb operation=\"walk\" path=\"result\" 2> /dev/null |grep result |wc -l` -ne 2 ]; then\n",
    "    echo 'creation failed, number of partitions <> 2'\n",
    "    exit -1\n",
    "fi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "f5f617ee-7bd2-4d4f-95a0-a8a7600c18a2",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "# delete result\n",
    "if [ `ipython cloud-object-store-housekeeping.ipynb 2> /dev/null | grep result |wc -l` -gt 0 ]; then\n",
    "    ipython cloud-object-store-housekeeping.ipynb operation=\"rm\" path=\"result\"\n",
    "fi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9f909de5-1165-4f44-bbcd-90d2b586397a",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "# verify result deleted\n",
    "if [ `ipython cloud-object-store-housekeeping.ipynb 2> /dev/null | grep result |wc -l` -gt 0 ]; then\n",
    "    echo 'deletion failed'\n",
    "    exit -1\n",
    "fi"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b659db1d-2b4b-4ed4-94fd-3d870da7409e",
   "metadata": {
    "tags": []
   },
   "source": [
    "# TEST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0bfeab55-3723-45bf-86be-0767e15efc1f",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "!ipython ibm-sql-query.ipynb out_format='parquet' out_exact_name='True'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3f3cbe1f-aeb4-4c71-92a4-ab12101e6470",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "# verify result existing and corrent number of partitions\n",
    "if [ `ipython cloud-object-store-housekeeping.ipynb operation=\"walk\" path=\"result\" 2> /dev/null |grep result |wc -l` -ne 1 ]; then\n",
    "    echo 'creation failed, number of results <> 1'\n",
    "    exit -1\n",
    "fi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "96725ac9-36e7-4ba7-b481-8e8a6002db3f",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "# delete result\n",
    "if [ `ipython cloud-object-store-housekeeping.ipynb 2> /dev/null | grep result |wc -l` -gt 0 ]; then\n",
    "    ipython cloud-object-store-housekeeping.ipynb operation=\"rm\" path=\"result\"\n",
    "fi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e109a936-d78e-4fc3-9580-d0c8e31e6bde",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "# verify result deleted\n",
    "if [ `ipython cloud-object-store-housekeeping.ipynb 2> /dev/null | grep result |wc -l` -gt 0 ]; then\n",
    "    echo 'deletion failed'\n",
    "    exit -1\n",
    "fi"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "fd31214a-ae75-44f7-a6a6-8cca0cb8a182",
   "metadata": {
    "tags": []
   },
   "source": [
    "# TEST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e609250e-2cd6-461a-9a73-934ec76aa525",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "!ipython ibm-sql-query.ipynb out_exact_name='True'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "18b8aae3-23b5-4825-93c8-80d385ef4d49",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "# verify result existing and corrent number of partitions\n",
    "if [ `ipython cloud-object-store-housekeeping.ipynb operation=\"walk\" path=\"result\" 2> /dev/null |grep result |wc -l` -ne 1 ]; then\n",
    "    echo 'creation failed, number of results <> 1'\n",
    "    exit -1\n",
    "fi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d48ef004-ed08-4d51-b9ef-0a9e7ce85a90",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "# delete result\n",
    "if [ `ipython cloud-object-store-housekeeping.ipynb 2> /dev/null | grep result |wc -l` -gt 0 ]; then\n",
    "    ipython cloud-object-store-housekeeping.ipynb operation=\"rm\" path=\"result\"\n",
    "fi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "3d938f6c-2b28-4414-ab05-1ef8b8318544",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "# verify result deleted\n",
    "if [ `ipython cloud-object-store-housekeeping.ipynb 2> /dev/null | grep result |wc -l` -gt 0 ]; then\n",
    "    echo 'deletion failed'\n",
    "    exit -1\n",
    "fi"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "85dfd4f1-e125-4ca0-bb55-77d71f3e1eff",
   "metadata": {
    "tags": []
   },
   "source": [
    "# TEST"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "88baeafe-7303-43bf-93ef-7433e2b1b3c6",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "!ipython ibm-sql-query.ipynb out_exact_name='True' out_no_jobid_folder='True'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "23a7f766-e695-4bcb-a0e2-cf04a88500f3",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "!ipython cloud-object-store-housekeeping.ipynb operation=\"walk\" path=\"result\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dac25036-7881-4b50-aab5-a9aebbee77a1",
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "# verify result existing and corrent number of partitions\n",
    "if [ `ipython cloud-object-store-housekeeping.ipynb operation=\"walk\" path=\"result\" 2> /dev/null |grep result |wc -l` -ne 1 ]; then\n",
    "    echo 'creation failed, number of results <> 1'\n",
    "    exit -1\n",
    "fi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1d8c917b-ef8c-4a50-b720-ca5617f7a583",
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "%%bash\n",
    "# delete result\n",
    "if [ `ipython cloud-object-store-housekeeping.ipynb 2> /dev/null | grep result |wc -l` -gt 0 ]; then\n",
    "    ipython cloud-object-store-housekeeping.ipynb operation=\"rm\" path=\"result\"\n",
    "fi"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "92d13a38-28b6-4bf8-b77b-397e6f6a56fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "%%bash\n",
    "# verify result deleted\n",
    "if [ `ipython cloud-object-store-housekeeping.ipynb 2> /dev/null | grep result |wc -l` -gt 0 ]; then\n",
    "    echo 'deletion failed'\n",
    "    exit -1\n",
    "fi"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.6"
  },
  "papermill": {
   "default_parameters": {},
   "duration": 470.538548,
   "end_time": "2021-03-22T20:37:13.369954",
   "environment_variables": {},
   "exception": null,
   "input_path": "/home/jovyan/work/examples/pipelines/pairs/component-library/transform/spark-csv-to-parquet.ipynb",
   "output_path": "/home/jovyan/work/examples/pipelines/pairs/component-library/transform/spark-csv-to-parquet.ipynb",
   "parameters": {},
   "start_time": "2021-03-22T20:29:22.831406",
   "version": "2.3.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
